# -*- coding: utf-8 -*-
import scrapy
import re
from myspider.items import sightitem

class JingQuSpider(scrapy.spiders.Spider):
    name = "jingqu"
    allowed_domains = ["www.ctrip.com", "you.ctrip.com"]
    start_urls = ["http://you.ctrip.com/sitemap/spotdis/c0"]

    def parse(self, response):
        baseurl = 'http://you.ctrip.com'
        sightlisturl = 'http://you.ctrip.com/sight/'
        beijing = sightlisturl + 'beijing1.html'
        tianjing = sightlisturl + 'tianjin154.html'
        shanghai = sightlisturl + 'shanghai2.html'
        chongqing = sightlisturl + 'chongqing158.html'
        hongkong = sightlisturl + 'hongkong38.html'
        macau = sightlisturl + 'macau39.html'
        #yield scrapy.Request(beijing, callback=self.parse_page)
        #yield scrapy.Request(tianjing, callback=self.parse_page)
        #yield scrapy.Request(shanghai, callback=self.parse_page)
        #yield scrapy.Request(chongqing, callback=self.parse_page)
        #yield scrapy.Request(hongkong, callback=self.parse_page)
        #yield scrapy.Request(macau, callback=self.parse_page)

        #四川
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100009', callback=self.parse_citylist)
        #山西c100056
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100056', callback=self.parse_citylist)
        #云南 c100007
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100007', callback=self.parse_citylist)
        #西藏 c100003
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100003', callback=self.parse_citylist)
        #陕西 c100057
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100057', callback=self.parse_citylist)
        #新疆 c100008
        #yield scrapy.Request( 'http://you.ctrip.com/sitemap/spotdis/c100008', callback=self.parse_citylist)

        #台湾
        #台北 http://you.ctrip.com/sightlist/taipeicity360.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/taipeicity360.html', callback=self.parse_page)
        #高雄 http://you.ctrip.com/sightlist/gaoxiong756.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/gaoxiong756.html', callback=self.parse_page)
        #花莲 http://you.ctrip.com/sightlist/hualien1366.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/hualien1366.html', callback=self.parse_page)
        #屏东 http://you.ctrip.com/sightlist/pingtung1364.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/pingtung1364.html', callback=self.parse_page)
        #台南 http://you.ctrip.com/sightlist/tainan757.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/tainan757.html', callback=self.parse_page)
        #南投市 http://you.ctrip.com/sightlist/nantou759.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/nantou759.html', callback=self.parse_page)
        #台中 http://you.ctrip.com/sightlist/taizhong1369.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/taizhong1369.html', callback=self.parse_page)
        #新北 http://you.ctrip.com/sightlist/newtaipeicity2063.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/newtaipeicity2063.html', callback=self.parse_page)
        #台东 http://you.ctrip.com/sightlist/taitung760.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/taitung760.html', callback=self.parse_page)
        #嘉义 http://you.ctrip.com/sightlist/chiayi758.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/chiayi758.html', callback=self.parse_page)
        #宜兰 http://you.ctrip.com/sightlist/yilan1383.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/yilan1383.html', callback=self.parse_page)
        #彰化 http://you.ctrip.com/sightlist/changhua3180.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/changhua3180.html', callback=self.parse_page)
        #基隆 http://you.ctrip.com/sightlist/keelung1382.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/keelung1382.html', callback=self.parse_page)
        #金门 http://you.ctrip.com/sightlist/jinmen1368.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/jinmen1368.html', callback=self.parse_page)
        #桃园市 http://you.ctrip.com/sightlist/taoyuan1439.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/taoyuan1439.html', callback=self.parse_page)
        #新竹 http://you.ctrip.com/sightlist/hsinchu1197.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/hsinchu1197.html', callback=self.parse_page)
        #澎湖 http://you.ctrip.com/sightlist/penghu1367.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/penghu1367.html', callback=self.parse_page)
        #嘉义县 http://you.ctrip.com/sightlist/chiayicounty3182.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/chiayicounty3182.html', callback=self.parse_page)
        #苗栗 http://you.ctrip.com/sightlist/miaoli3199.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/miaoli3199.html', callback=self.parse_page)
        #马祖 http://you.ctrip.com/sightlist/matsu1445356.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/matsu1445356.html', callback=self.parse_page)
        #云林市 http://you.ctrip.com/sightlist/yunlin3197.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/yunlin3197.html', callback=self.parse_page)
        #新竹县 http://you.ctrip.com/sightlist/hsinchucounty3198.html
        yield scrapy.Request('http://you.ctrip.com/sightlist/hsinchucounty3198.html', callback=self.parse_page)




        #sitemaps = response.xpath('//div[@class="map_title cf"]/a/@href').extract()
        #for sitemap in sitemaps:
        #   yield scrapy.Request( baseurl + sitemap, callback=self.parse_citylist)
        #   pass

        pass

    def parse_citylist(self, response):
        baseurl = "http://you.ctrip.com"
        links = response.xpath('//ul[@class="map_linklist cf"]/li/a/@href').extract()
        for link in links:
            link = link.replace('sightlist/', 'sight/')
            link = link.replace('countrysightlist/', 'sight/')
            link = link.replace('countrysight/', 'sight/')
            print("#########", link)
            yield scrapy.Request(baseurl + link, callback=self.parse_page)
        pass

    def parse_page(self, response):
        baseurl = "http://you.ctrip.com"
        sightselectors = response.xpath('//div[@class="list_wide_mod2"]//dt')

        for selector in sightselectors:
            url = baseurl + selector.xpath('a/@href').extract_first()
            rank = selector.xpath('s/text()').extract_first()
            rankingnum = 0
            if not rank is None:
                rankingstr = re.sub(r'[^\x00-\x7f]', '', rank)
                rankingnum = int(rankingstr)
            yield scrapy.Request(url, callback=lambda response, ranking=rankingnum:self.parse_sight(response, ranking), dont_filter=True)
            #yield scrapy.Request(url, callback=self.parse_imgcount)
            pass
        nextpage = response.xpath('//a[@class="nextpage"]/@href').extract_first()
        if not nextpage is None:
            if nextpage.strip() != '':
                yield scrapy.Request(baseurl + nextpage, callback=self.parse_page)
        pass

    def parse_sight(self, response, ranking):
        # 提取内容
        name = response.xpath('//div[@class="dest_toptitle detail_tt"]//a/text()').extract_first()
        comments = response.xpath('//span[@class="f_orange"]/text()').extract_first()
        if not name is None:
            name = name.strip()
        address = response.xpath('//p[@class="s_sight_addr"]/text()').extract_first()
        if not address is None:
            address = address.strip().replace('地址：','')
        desc = response.xpath('//div[@itemprop="description"]').extract_first()
        if desc is None:
            print('###', 'desc is none')
            return
        desc = desc.strip()
        if desc == '':
            print('###', 'desc is none')
            return
        province = response.xpath('//div[@class="breadbar_v1 cf"]/ul/li[4]/a/text()').extract_first()
        if not province is None:
            province = province.strip()
        if province == '':
            print('###', 'province is none')
            return
        imgurls = response.xpath('//div[@class="carousel-inner"]//img/@src').extract()
        imgcount = 0
        if not imgurls is None:
           imgcount = len(imgurls)

        infoarray =  response.xpath('//ul[@class="s_sight_in_list"]//span/text()').extract()

        info =[]
        index = 0
        while(index < len(infoarray)-1):
            key = infoarray[index]
            key = re.sub(r"\xa0", "", key)
            value = infoarray[index+1]
            value = re.sub(r"\xa0", "", value)
            if value.strip()!='':
                info.append({ key.strip(): value.strip()})
            index += 2
        pass

        key = response.xpath('//dl[@class="s_sight_in_list"]//dt/text()').extract_first()
        if not key is None:
            key = key.strip()
            key = re.sub(r"\xa0", "", key)
        value = response.xpath('//dl[@class="s_sight_in_list"]//dd/text()').extract_first()
        if not value is None:
            value = value.strip()
            value = re.sub(r"\xa0", "", value)
            if value.strip()!='':
                info.append({ key: value})
                pass

        item = sightitem()
        item['name'] = name
        item['province'] = province
        item['areaname'] = self.get_area_name(address)
        item['address'] = address
        item['desc'] = self.clear_html_re(desc)
        item['info'] = info
        item['ranking'] = ranking
        item['comments'] = comments
        item['imgcount'] = imgcount
        yield item

    def parse_imgcount(self, response):
        # 提取内容
        name = response.xpath('//div[@class="dest_toptitle detail_tt"]//a/text()').extract_first()
        if not name is None:
            name = name.strip()
        address = response.xpath('//p[@class="s_sight_addr"]/text()').extract_first()
        if not address is None:
            address = address.strip().replace('地址：','')
        desc = response.xpath('//div[@itemprop="description"]').extract_first()
        if desc is None:
            return
        desc = desc.strip()
        if desc == '':
            return
        province = response.xpath('//div[@class="breadbar_v1 cf"]/ul/li[4]/a/text()').extract_first()
        if not province is None:
            province = province.strip()
        if province == '':
            return
        imgurls = response.xpath('//div[@class="carousel-inner"]//img/@src').extract()
        imgcount = 0
        if not imgurls is None:
            imgcount = len(imgurls)

        if imgcount>0:
            return

        item = sightitem()
        item['name'] = name
        item['address'] = address
        item['imgcount'] = imgcount
        yield item

    def clear_html_re(self, src_html):
        content = re.sub(r"</?(.+?)>", "", src_html) # 去除标签
        dst_html = re.sub(r"\s+", "", content)  # 去除空白字符
        return dst_html
        pass

    def get_area_name(self, address):
        p = re.compile(r'[\u4e00-\u9fa5]{2,5}?(?:区|县)')
        p1 = re.compile(r'[\u4e00-\u9fa5]{2,5}?(?:省|自治区|市|区|县)')
        areaname = ''
        if len(p.findall(address))>0:
            address = p.findall(address)[0]
            areas = p1.findall(address)
            if not areas is None:
                areaname = areas[len(areas)-1]

        return areaname
    pass